{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://www.jianshu.com/c/bDHhpK\n",
    "\n",
    "https://www.jianshu.com/c/bDHhpK?order_by=added_at&page=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lxml import etree\n",
    "import requests\n",
    "import xlwt\n",
    "import time\n",
    "import pymongo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#,'x-requested-with':'XMLHttpRequest','x-csrf-token':'LdG/fpHm2nV502Kg+TNaZw7FZahT0+7wMarS8/MhZ8UsP0+TEBNN7xfHKlnTDiLfrImEfG/t0pVRRqrJXR+W9A=='\n",
    "def get_res(url):\n",
    "    headers={'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'}\n",
    "    response=requests.get(url,headers=headers)\n",
    "    print(response.text)\n",
    "    return response.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = pymongo.MongoClient('localhost', 27017)\n",
    "mydb = client['mydb']\n",
    "table_jianshu = mydb['jianshu']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_info():\n",
    "    format_url='https://www.jianshu.com/c/bDHhpK?order_by=added_at&page={}'\n",
    "    # 前10页\n",
    "    urls=[format_url.format(str(i)) for i in range(1,2)]\n",
    "    for url in urls:\n",
    "        selector=etree.HTML(get_res(url))\n",
    "        infos=selector.xpath('//*[@id=\"list-container\"]/ul/li')\n",
    "        for info in infos:\n",
    "            \n",
    "            title=info.xpath('div/a/text()')[0].strip()\n",
    "            author=info.xpath('div/div/a[1]/text()')[0].strip()\n",
    "#             comments=info.xpath('div/div/a[2]/text()')\n",
    "#             print(comments)\n",
    "            comment=info.xpath('div/div/a[2]/text()')[1].strip()\n",
    "            like=info.xpath('div/div/span/text()')[0].strip()\n",
    "            imgs=info.xpath('a/img/@src')\n",
    "            for img in imgs:\n",
    "                fix_img=img\n",
    "            img_fix='http://upload-images.jianshu.io/'+fix_img[2:]\n",
    "            info_list={'title':title,'author':author,'comment':comment,'like':like,'img':img_fix}\n",
    "            \n",
    "            table_jianshu.insert_one(info_list)\n",
    "#             all_info_list.append(info_list)\n",
    "        time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = pymongo.MongoClient('localhost', 27017)\n",
    "mydb = client['mydb']\n",
    "timeline = mydb['timeline']\n",
    "\n",
    "def get_time_info(url,page):\n",
    "    user_id = url.split('/')\n",
    "    user_id = user_id[4]\n",
    "    if url.find('page='):\n",
    "        page = page+1\n",
    "    html = get_res(url)\n",
    "    selector = etree.HTML(html.text)\n",
    "    infos = selector.xpath('//ul[@class=\"note-list\"]/li')\n",
    "    for info in infos:\n",
    "        dd = info.xpath('div/div/div/span/@data-datetime')[0]\n",
    "        type = info.xpath('div/div/div/span/@data-type')[0]\n",
    "        timeline.insert_one({'date':dd,'type':type})\n",
    "\n",
    "    id_infos = selector.xpath('//ul[@class=\"note-list\"]/li/@id')\n",
    "    if len(infos) > 1:\n",
    "        feed_id = id_infos[-1]\n",
    "        max_id = feed_id.split('-')[1]\n",
    "        next_url = 'http://www.jianshu.com/users/%s/timeline?max_id=%s&page=%s' % (user_id, max_id, page)\n",
    "        get_time_info(next_url, page)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_time_info('http://www.jianshu.com/users/9104ebf5e177/timeline',1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
